#First compute the YaTa variable
setwd("~/accept_decline")

mean_impute = function(x)
{
	mu = mean(na.omit(x))
	x[is.na(x)] = mu
	x
}

df = readRDS("train.rds")
df = data.frame(lapply(df, mean_impute))
df = data.frame(df, loss_ind=df$loss>0)
#m=glm(formula = loss_ind ~ f527 + f528, family = binomial(), data = df)
#pred_yata = predict(m, type="response")
#yata variable replaced by tantrev variable
m=glm(formula = loss_ind ~ f274 + f528, family = binomial(), data = df)
pred_tantrev = predict(m, type="response")

a = cut(pred_tantrev, quantile(pred_tantrev, seq(0, 1, .01), include.lowest=T))
plot(tapply(df$loss_ind, a, mean))

sum(df$loss_ind[pred_tantrev>=.09])/sum(df$loss_ind)

df2 = df[pred_tantrev>.09,]

#df2 = df[pred_yata>.09,]

quantilize = function(x)
{
	qntls = unique(quantile(x, seq(0, 1, .05)))
	print(length(qntls))
	if(length(qntls)>1)cut(x, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(x)))
}


df_quantile = lapply(df2[,2:(ncol(df2) - 2)], quantilize)
df_quantile = data.frame(df_quantile, loss_ind=df2$loss_ind, loss=df2$loss)
m_fine = glm(loss_ind ~ f13 + f67 + f84 + f404 + f514+ f527 + f201 + f211 + f238, data=df_quantile, family=binomial)
#this model should include the tantrev variable


pred = predict(m_fine, type="response")
plot(quantile(pred, seq(0, 1, .02)), type="l")
a=cut(pred, quantile(pred, seq(0, 1, .02), include.lowest=T))
points(tapply(df_quantile$loss_ind, a, mean))
tapply(df_quantile$loss, a, median)



##Now need to apply to test, which means test variables must be quantilized

quantilize2 = function(x,y)
{
	qntls = unique(quantile(x, seq(0, 1, .05)))
	print(length(qntls))
	if(length(qntls)>1)cut(y, qntls, 1:(length(qntls) - 1), include.lowest=T)
	else as.factor(rep(1, length(y)))
}

df_test = readRDS("test.rds")
df_test = data.frame(lapply(df_test, mean_impute))
#pred_yata_test = predict(m, newdata=df_test, type="response")
pred_tantrev_test = predict(m, newdata=df_test, type="response")
df_test2 = df_test[pred_tantrev_test>.09,]

#f13 + f67 + f84 + f404 + f514+ f527 + f201 + f211 + f238

df_test_quantile = data.frame(id=df_test2$id, 
f13=quantilize2(df2$f13, df_test2$f13),
f67=quantilize2(df2$f67, df_test2$f67),
f84=quantilize2(df2$f84, df_test2$f84),
f404=quantilize2(df2$f404, df_test2$f404),
f514=quantilize2(df2$f514, df_test2$f514),
f527=quantilize2(df2$f527, df_test2$f527),
f201=quantilize2(df2$f201, df_test2$f201),
f211=quantilize2(df2$f211, df_test2$f211),
f238=quantilize2(df2$f238, df_test2$f238))

pred_test = predict(m_fine, type="response", newdata = df_test_quantile)
test_loss = rep(0, nrow(df_test2))

(0.506,0.518] (0.518,0.529] (0.529,0.541] 
            1             0             1             1             1 
(0.541,0.552] (0.552,0.563] (0.563,0.572] (0.572,0.584] (0.584,0.594] 
            1             1             1             1             1 
(0.594,0.603] (0.603,0.612]  (0.612,0.62]   (0.62,0.63]  (0.63,0.639] 
            2             2             2             2             2 
(0.639,0.647] (0.647,0.656] (0.656,0.664] (0.664,0.674] (0.674,0.683] 
            3             2             3             2             3 
(0.683,0.691] (0.691,0.701]  (0.701,0.71]  (0.71,0.719] (0.719,0.729] 
            3             3             2             4             4 
(0.729,0.739] (0.739,0.749] (0.749,0.759] (0.759,0.769] (0.769,0.782] 
            3             4             4             5             5 
(0.782,0.795]  (0.795,0.81]  (0.81,0.827] (0.827,0.851] (0.851,0.926] 
            5             6             6             7             8 
            
				(0.473,0.484]  (0.484,0.495]  (0.495,0.507]  (0.507,0.517] 
           0.0            1.0            0.0            1.0            0.0 
 (0.517,0.529]   (0.529,0.54]   (0.54,0.551]   (0.551,0.56]   (0.56,0.571] 
           1.0            1.0            1.0            1.5            1.0 
  (0.571,0.58]    (0.58,0.59]   (0.59,0.598]  (0.598,0.606]  (0.606,0.615] 
           2.0            2.0            2.0            2.0            2.0 
 (0.615,0.625]  (0.625,0.633]  (0.633,0.642]  (0.642,0.651]   (0.651,0.66] 
           2.0            2.0            2.0            2.0            2.5 
   (0.66,0.67]   (0.67,0.678]  (0.678,0.687]  (0.687,0.696]  (0.696,0.705] 
           3.0            3.0            3.0            3.0            3.5 
 (0.705,0.715]  (0.715,0.725]  (0.725,0.736]  (0.736,0.747]  (0.747,0.759] 
           3.0            4.0            4.0            4.0            4.0 
 (0.759,0.771]  (0.771,0.786]  (0.786,0.804]  (0.804,0.827]  (0.827,0.913] 
           5.0            6.0            5.0            6.0            8.0 



test_loss[pred_test>.517 & pred_test<=.571] = 1
test_loss[pred_test>.571 & pred_test<=.66] =2
test_loss[pred_test>.66 & pred_test<=.705] = 3
test_loss[pred_test>.705 & pred_test<=.759] = 4
test_loss[pred_test>.759 &pred_test<=.795] = 5
test_loss[pred_test>.759 &pred_test<=.804] = 5
test_loss[pred_test>.804 &pred_test<=.827] = 6
test_loss[pred_test>.827] = 8

df_ans = data.frame(id=df_test2$id, test_loss)
df_ans2 = merge(data.frame(id=df_test$id), df_ans, by="id", all.x=T)
names(df_ans2)=c("id","loss")
df_ans2$id[is.na(df_ans2$id)]=0

write.csv(df_ans2,"submission.csv", row.names=F)




